package jude.java.money.scan;

import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import jude.java.framework.beans.Regexp;
import jude.java.money.beans.Site;
import jude.java.money.service.ScanService;
import jude.java.money.spring.Services;
import jude.java.money.task.SingleTask;
import jude.java.util.WebUtils;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.lang.StringUtils;

/**
 * 在指定站点下的站点扫描
 */
public class LimitSiteScaner extends SingleTask implements Regexp.Listener {
	private static final int MAX_SIZE = 1024 * 1024;		//1 M
	
//	private static final String HTTP_URL_PATTERN = "<a\\b[^>]*href=['\\\"](https?://[\\w-/\\\\\\.=?&]{4,100})['\\\"]";
	public static final String[] URL_PATTERN = {
		"<a\\b[^>]*href=['\"]\\s*([\\w./?=&-:;%]{4,200})['\"]"
	};
	
	public static final int PATTERN_TYPE_RELATIVE = 1;
	public static final int PATTERN_TYPE_SITE = 2;
	
	private static final String DOMAIN_PATTERN = "http://(?:(www|bbs|blog)\\.)?([^\\\\/:?]{4,200})";
	
//	private String site;
	private int max;
	private HttpClient client;
	private HashMap<String, String> pending;
	private HashMap<String, String> finished;
	private ScanService service;
	private char[] buffer;
	private int count;
	private boolean stop;		//停止
	private boolean denied;		//拒绝访问, 如需要输入验证码
	
	public LimitSiteScaner(String... urls) {
//		this.site = site;
		service = Services.getScanService();
		buffer = new char[4096];
		pending = new HashMap<String, String>(100);
		finished = new HashMap<String, String>(50);
		for (int i = 0; i < urls.length; i++) {
			pending.put(getKey(urls[i]), urls[i]);
		}
		initHttpClient();
	}
	
	@Override
	protected void execute(Object obj) {
		while (!stop && !denied && !pending.isEmpty() && (max <= 0 || count < max)) {
			Map.Entry<String, String> e = pending.entrySet().iterator().next();
			String key = e.getKey();
			pending.remove(key);
			String url = e.getValue();
			finished.put(key, null);
			next(url, true);
		}
	}
	
	private void next(String url, boolean retry) {
		try {
			System.out.println(url);
			//url 须是合法的url
			GetMethod get = new GetMethod(url);
			get.setFollowRedirects(true);
			WebUtils.pretendRequestHeader(get);
			int statusCode = client.executeMethod(get);
			if (statusCode == HttpStatus.SC_OK) {
				handle(url, get);
//			} else {
//				System.out.println(statusCode + " " + url);
			}
		} catch (org.apache.commons.httpclient.InvalidRedirectLocationException e){
			//give up
		} catch (org.apache.commons.httpclient.CircularRedirectException e) {
			//give up
		} catch (javax.net.ssl.SSLHandshakeException e) {
			//give up
		} catch (org.apache.commons.httpclient.ConnectTimeoutException e) {
			//give up
		} catch (java.net.UnknownHostException e) {
			//give up
		} catch (ThreadDeath e) {
			//give up
			throw e;
		} catch (Throwable e) {
			// other exception, retry
			initHttpClient();
			if (retry) {
				next(url, false);
			} else {
				e.printStackTrace();
			}
		}
	}
	
	private static String normalize(String url) {
		if (StringUtils.isEmpty(url))
			return null;
		try {
			url = new URI(url).normalize().toString();
			int n = url.indexOf('#');
			if (n >= 0)
				url = url.substring(0, n);
			if (url.indexOf("/../") != -1)
				return null;
			return url;
		} catch (URISyntaxException e) {
		}
		return null;
	}
	
	/**
	 * 判断是否被中断, 输入验证码之类的
	 * */
	protected boolean denied(StringBuilder content) {
		return false;
	}
	
	protected Regexp[] getPatterns() {
		return null;
	}
	
//	/**
//	 * 获取地址的正则表达式, 第一个匹配为域名
//	 * */
//	protected String[] getUrlPatterns() {
//		return URL_PATTERN;
//	}
//	
//	protected int[] getPatternTypes() {
//		return new int[] {PATTERN_TYPE_RELATIVE | PATTERN_TYPE_SITE};
//	}
	
	@Override
	public boolean found(Regexp regexp, String str) {
		int type = regexp.getType();
		System.out.println(type + "\t" + str);
		if (type == PATTERN_TYPE_RELATIVE) {
			foundRelativeUrl(str);
		} else if (type == PATTERN_TYPE_SITE) {
			if (decodeUrl())
				try {
					str = URLDecoder.decode(str, "UTF-8");
				} catch (UnsupportedEncodingException e) {
				}
			found(str);
		}
		return true;
	}
	
	private boolean handle(String base, HttpMethodBase method) throws IOException {
		StringBuilder sb = new StringBuilder();
		Reader reader = new InputStreamReader(method.getResponseBodyAsStream(), "ISO-8859-1");
		int n;
		while ((n = reader.read(buffer)) != -1) {
			sb.append(buffer, 0, n);
			if (sb.length() >= MAX_SIZE)
				break;
		}
		if (sb.length() == 0)
			return false;
		if (denied(sb)) {
			denied = true;
			return false;
		}
		Regexp[] ps = getPatterns();
		if (ps == null)
			return false;
		for (int i = 0; i < ps.length; i++) {
			if (ps[i] != null)
				ps[i].search(sb.toString(), this);
		}
		
//		String[] patterns = getUrlPatterns();
//		int[] types = getPatternTypes();
//		for (int i = 0; i < patterns.length; i++) {
//			int start = 0;
//			Pattern p = Pattern.compile(patterns[i], PATTERN_FLAGS);
//			Matcher m = p.matcher(sb);
//			while (m.find(start)) {
//				String url = m.group(1);
//				url = getUrl(base, url);
//				if (url != null) {
//					if (types[i] == PATTERN_TYPE_RELATIVE) {
//						foundRelativeUrl(url);
//					} else if (types[i] == PATTERN_TYPE_SITE){
//						found(url);
//					}
//				}
//				start = Math.max(start + 1, m.end(0));
//			}
//		}
		return true;
	}
	
	protected boolean decodeUrl() {
		return false;
	}
	
	protected String getKey(String relativeUrl) {
		return relativeUrl;
	}
	
	private void foundRelativeUrl(String url) {
		if (url == null)
			return;
		String key = getKey(url);
		if (pending.containsKey(key) || finished.containsKey(key))
			return;
//		System.out.println("\t\t" + key + "\t" + url);
		pending.put(key, url);
	}
	
	protected String getUrl(String base, String url) {
		char c = url.charAt(0);
		if (c == '#' || c == '?')
			return null;
		if (url.startsWith("http://")) {// || url.startsWith("https://")
			return normalize(url);
		}
		
		//ignore like: javascript:void(0), ftp://abc.cn
		//support: /?a=b:c
		int q = url.indexOf('?');
		int u = url.indexOf(':');
		if (q == -1 && u >= 0)
			return null;
		if (q > 0 && u >= 0 && u < q)
			return null;
		
		q = base.indexOf('?', 7);
		if (q > 0) {
			base = base.substring(0, q);
		}
		q = base.indexOf("#", 7);
		if (q > 0) {
			base = base.substring(0, q);
		}
		if (c == '/' || c == '\\') {
			//绝对路径
			int n = base.indexOf('/', 7);	//"http://".length()
			if (n > 0) {
				url = base.substring(0, n) + url;
			} else {
				url = base + url;
			}
		} else {
			//相对路径
			int n = base.indexOf('/', 7);	//"http://".length()
			if (n > 0) {
				n = base.lastIndexOf('/');
				url = base.substring(0, n + 1) + url;
			} else {
				url = base + '/' + url;
			}
		}
		return normalize(url);
	}
	
	private void found(String url) {
		if (url == null)
			return;
//		if (pending.contains(url) || finished.contains(url))
//			return;
//		if (site != null && url.startsWith(site)) {
//			pending.add(url);
//			return;
//		}
		Pattern p = Pattern.compile(DOMAIN_PATTERN, Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(url);
		if (m.find()) {
			String domain = m.group(m.groupCount());
			if (service.save(new Site(domain))) {
				count ++;
				System.out.println("\t" + count + " " + domain);
			}
		}
	}
	
	private void initHttpClient() {
		client = new HttpClient();
		client.getHttpConnectionManager().getParams().setConnectionTimeout(20000);
		client.getHttpConnectionManager().getParams().setSoTimeout(60000);
	}
}
